package com.chance.cc.crawler.development.bootstrap.xcar.xcarSeriesNewsStart;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.record.CrawlerResultRecord;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import com.chance.cc.crawler.meta.core.bean.crawler.CrawlerDomainUrls;
import org.apache.commons.lang.StringUtils;
import org.junit.Test;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @ClassName XcarNewsArticleClass
 * @Description TODO
 * @Author songding
 * @Date 2021/8/12 10:09
 * @Version 1.0
 * 爱卡汽车   车系文章板块采集
 * 30天文章链接累计  和 主贴 互动量  全量回复
 **/
public class XcarSeriesNewsStart {
    private static final String DOMAIN = "xcar";
    private static final String SIDE = "SN";
    private static Proxy proxy = new Proxy();
    static{
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("H7307T4706B25G4D");
        proxy.setPassword("05B4877CC39192C0");

    }
    public static void main(String[] args) {
      // xcarArticleNew();
       //xcarArticle();
        Article();
        //mysqlToScript();
    }
    //关键字传入
    public static void xcarArticleNew() {
        String url = "https://newcar.xcar.com.cn/";
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpConfig(HttpConfig.me(DOMAIN))
                .httpUrl(url)
                .domain(DOMAIN)
                .proxy(proxy)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 7, null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                //控制链接的累计
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .build();
        record.tagsCreator().bizTags().addDomain(DOMAIN);
        record.tagsCreator().bizTags().addSite(SIDE);
        record.tagsCreator().bizTags().addSiteBiz(SIDE);
        record.getHttpRequest().setResponseCharset("UTF-8");
        //创建 评论信息的过滤条件
        CrawlerRecord filterCrawlerRecord =  new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(DOMAIN));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null));
        //以json转换保证数据传输的完整性
        record.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        // TODO   关键字传入设置
        CrawlerRequestRecord keywordRecord = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN,turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/xcar/keys?site=carSeries")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(DOMAIN, DOMAIN, System.currentTimeMillis(), DOMAIN)
                .crawlerThreadNum(20)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(DOMAIN))
                .consoleResultPipeline("kafka")
                .consoleResultPipeline("redis")
                .fileResultPipeline("redis", "F:\\chance_log\\xcar\\article\\url.log", false)
                .fileResultPipeline("kafka", "F:\\chance_log\\xcar\\article\\Txt.log", false)
                .requestRecord(record)
                //.supportRecord(keywordRecord)
                .build("com.chance.cc.crawler.development.scripts.xcar")
                .start();


    }
    //单独清洗文章
    private static void Article() {
        //https://newcar.xcar.com.cn/201910/news_2043238_1.html三种文章格式
        //https://info.xcar.com.cn/201907/news_2040523_1.html
        //https://drive.xcar.com.cn/201905/news_2038827_1.html
        String url = "https://info.xcar.com.cn/202110/news_2062623_1.html";
        //https://info.xcar.com.cn/202109/news_2062144_1.html?viewtype=all
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpConfig(HttpConfig.me(DOMAIN))
                .httpUrl(url)
                .domain(DOMAIN)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*62,null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                //控制链接的累计
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .proxy(proxy)
                .build();
        record.setNeedWashPage(true);
        record.tagsCreator().bizTags().addDomain(DOMAIN);
        record.tagsCreator().bizTags().addSite(SIDE);
        record.tagsCreator().bizTags().addSiteBiz(SIDE);

        //创建 评论信息的过滤条件
        CrawlerRecord filterCrawlerRecord =  new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(DOMAIN));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*62,null));
        //以json转换保证数据传输的完整性
        record.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));


        DevCrawlerController.builder()
                .triggerInfo(DOMAIN,DOMAIN,System.currentTimeMillis(),DOMAIN)
                .crawlerThreadNum(10)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(DOMAIN))
                .consoleResultPipeline("kafka")
                //.consoleResultPipeline("redis")
                //.fileResultPipeline("redis","F:\\chance_log\\ak\\seriesArticle\\addArticleUrl.log",false)
                .fileResultPipeline("kafka","F:\\chance_log\\ak\\seriesArticle\\comment.log",false)
                .requestRecord(record)
                .build("com.chance.cc.crawler.development.scripts.xcar")
                .start();

    }

    //根据单个车系进行  清洗文章内容  和  累计链接
    public static void xcarArticle(){
        //https://newcar.xcar.com.cn/
        String url = "https://newcar.xcar.com.cn/404/";
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpConfig(HttpConfig.me(DOMAIN))
                .httpUrl(url)
                .domain(DOMAIN)
                .proxy(proxy)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*30,null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();
        record.tagsCreator().bizTags().addDomain(DOMAIN);
        record.tagsCreator().bizTags().addSite(SIDE);
        record.tagsCreator().bizTags().addSiteBiz(SIDE);

        //创建 评论信息的过滤条件
        CrawlerRecord filterCrawlerRecord =  new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(DOMAIN));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null));
        //以json转换保证数据传输的完整性
        record.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));

        DevCrawlerController.builder()
                .triggerInfo(DOMAIN,DOMAIN,System.currentTimeMillis(),DOMAIN)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(DOMAIN))
                .consoleResultPipeline("kafka")
                .consoleResultPipeline("redis")
                .fileResultPipeline("redis","F:\\chance_log\\xcar\\article\\addArticleUrl.log",false)
                .fileResultPipeline("kafka","F:\\chance_log\\xcar\\article\\ArticleTxt.log",false)
                .requestRecord(record)
                .build("com.chance.cc.crawler.development.scripts.xcar")
                .start();
    }

  public static void mysqlToScript(){
      String url = "https://newcar.xcar.com.cn/";
      CrawlerRequestRecord record = CrawlerRequestRecord.builder()
              .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
              .httpConfig(HttpConfig.me(DOMAIN))
              .httpUrl(url)
              .domain(DOMAIN)
              .releaseTime(System.currentTimeMillis())
              .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
              .needParsed(false)
              .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 60, null))
              .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
              .build();

      record.tagsCreator().bizTags().addDomain(DOMAIN);
      record.tagsCreator().bizTags().addSite(SIDE);
      record.tagsCreator().bizTags().addSiteBiz(SIDE);

      CrawlerRequestRecord initCommentCrawlerRecord = CrawlerRequestRecord.builder()
              .startPageRequest(DOMAIN,turnPageItem)
              .httpUrl("http://192.168.1.215:9599/crawler/domain/common/api/v1/"
                      +DOMAIN+"/results/queue/init?requestQueueName=crawler_result_queue_XCAR_ARTICLE&site="+SIDE+"&hourFromNow=170")
              .requestLabelTag(supportSource)
              .requestLabelTag(internalDownload)
              .build();
      //initCommentCrawlerRecord.tagsCreator().bizTags().addCustomKV("domain_result_json","domain_result_json");
      HttpRequestBody jsonBody = HttpRequestBody.json(JSON.toJSONString(record), "utf-8");
      initCommentCrawlerRecord.getHttpRequest().setMethod(HttpConstant.Method.POST);
      initCommentCrawlerRecord.getHttpRequest().setRequestBody(jsonBody);


      DevCrawlerController.builder()
              .triggerInfo(DOMAIN,DOMAIN,System.currentTimeMillis(),DOMAIN)
              .crawlerRequestQueue(DevCrawlerController.devRequestQueue(DOMAIN))
              .consoleResultPipeline()
              .requestRecord(record)
              .supportRecord(initCommentCrawlerRecord)
              .build("com.chance.cc.crawler.development.scripts.xcar")
              .start();
  }

    @Test
    public void push() throws IOException {
        String path = "F:\\chance_log\\xcar\\article\\addArticleUrl.log";
        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);

        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
            String readLine = null;
            while((readLine = bufferedReader.readLine()) != null){
                if (StringUtils.isBlank(readLine)){
                    continue;
                }
                /*
                *将crawlerData数据格式转换成 crawlerResult数据格式
                * */
                CrawlerResultRecord crawlerResultRecord = new CrawlerResultRecord();
                String[] split = readLine.split("\t");
                crawlerResultRecord.setJsonStr(readLine);
                redisReader.listPush("crawler_result_queue_XCAR_ARTICLE", JSON.toJSONString(crawlerResultRecord));
               // pop();
            }
            System.out.println(redisReader.llen("crawler_result_queue_XCAR_ARTICLE"));
            HttpClientDownloader downloader = new HttpClientDownloader();
            downloader.setClientGenerator(new HttpClientGenerator());
            downloader.setProxyProvider(new SiteConfigProxyProvider());
            HttpConfig httpConfig = HttpConfig.me(DOMAIN);
            com.chance.cc.crawler.core.downloader.HttpRequest httpRequest = new HttpRequest();
            httpRequest.setResponseCharset("UTF-8");
            httpRequest.setUrl("http://192.168.1.215:9599/crawler/domain/common/api/v1/"+DOMAIN+"/search/results/sync?"
                    + "resultQueue=crawler_result_queue_XCAR_ARTICLE&site="+SIDE);
            downloader.download(httpRequest,httpConfig);
        } catch (Exception e) {
            e.printStackTrace();
        }//http://192.168.1.215:9599/crawler/domain/common/api/v1/xcar/search/results/sync?resultQueue=crawler_result_queue_XCAR_ARTICLE&site=SN

    }

    @Test
    public void txt(){
        String url = "https://newcar.xcar.com.cn/";
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpConfig(HttpConfig.me(DOMAIN))
                .httpUrl(url)
                .domain(DOMAIN)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 60, null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();

        HttpRequestBody jsonBody = HttpRequestBody.json(JSON.toJSONString(record), "utf-8");
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpConfig httpConfig = HttpConfig.me(DOMAIN);

        com.chance.cc.crawler.core.downloader.HttpRequest httpRequest = new HttpRequest();

        httpRequest.setResponseCharset("UTF-8");
        httpRequest.setMethod(HttpConstant.Method.POST);
        httpRequest.setRequestBody(jsonBody);
        httpRequest.addHeader("crawler_schedule_tag","crawler_schedule_tag");
        httpRequest.setUrl("http://192.168.1.215:9599/crawler/domain/common/api/v1/"+DOMAIN+
                "/results/queue/init?requestQueueName=crawler_result_queue_XCAR_ARTICLE&site="+SIDE+"&hourFromNow=10000");
        HttpPage download = downloader.download(httpRequest, httpConfig);

    }
    @Test
    public void redisToDev(){
        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);
        Long a = redisReader.llen("crawler_result_queue_XCAR_ARTICLE_list");
        for (int i = 0; i < a ; i++) {
            CrawlerRequestRecord crawlerRequestRecord = redisReader.recordPop("crawler_result_queue_XCAR_ARTICLE_list", "crawler_result_queue_XCAR_ARTICLE_map");
            KVTag url1 = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("domain_result_json");
            CrawlerDomainUrls crawlerDomainUrls = JSON.parseObject(String.valueOf(url1.getVal()), CrawlerDomainUrls.class);
            String url = crawlerDomainUrls.getUrl();
            JSONObject jsonObject = JSONObject.parseObject(url);
            String s = (String) jsonObject.get("url");
            Object releaseTime = jsonObject.get("releaseTime");
            Long.valueOf((String) releaseTime);

        }

    }





    @Test
    public void s(){
        String s = "活动时间：2021-08-13 至 2021-09-11";
        String authorId = "//my.xcar.com.cn/space.php?uid=6717082";
        String url = "//https://comment.xcar.com.cn/comment.php?nid=2052070&ctype=0";
        String common = "news_2054655_1.html";
        System.out.println(common.substring(5, common.length() - 7));
        String id = url.substring(46,url.length()-8);
        System.out.println(id);
        //String[] split = authorId.split("//my.xcar.com.cn/space.php\\?uid=");
       //authorId = split[1];
        //String[] split = s.split("：");
        //System.out.println(split[0]+"*"+split[1]);
       // System.out.println(authorId);
        //System.out.println(split[1].substring(0,10));

        String pageUrl ="https://comment.xcar.com.cn/interface/index.php?iact=CommentLevel&cid="+2061182
                +"&action=getNewsComment&sort=time&ctype=0&page="+(Integer.valueOf("1")+1)+"&limit=25&_="+System.currentTimeMillis();
        String o = "https://comment.xcar.com.cn/interface/index.php?iact=CommentLevel&cid=2061182" +
                "&action=getNewsComment&sort=time&ctype=0&page=1&limit=25&_=1629093298172";
        String url1    = "https://comment.xcar.com.cn/interface/index.php?iact=CommentLevel&cid=2061182" +
                "&action=getNewsComment&sort=ups&ctype=0&page=1&limit=25&_=1629093955746";
        System.out.println(pageUrl);
    }
}
